{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(1202698:140225058071744,MainProcess):2024-04-26-15:55:14.685.99 [mindspore/run_check/_check_version.py:102] MindSpore version 2.3.0rc1 and cuda version 12.2.140 does not match, CUDA version [['10.1', '11.1', '11.6']] are supported by MindSpore officially. Please refer to the installation guide for version matching information: https://www.mindspore.cn/install.\n",
      "/home/daiyuxin/miniconda3/envs/lyk_ms2.3/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.622 seconds.\n",
      "Prefix dict has been built successfully.\n",
      "[WARNING] ME(1202698:140225058071744,MainProcess):2024-04-26-15:55:16.674.899 [mindspore/run_check/_check_version.py:102] MindSpore version 2.3.0rc1 and cuda version 12.2.140 does not match, CUDA version [['10.1', '11.1', '11.6']] are supported by MindSpore officially. Please refer to the installation guide for version matching information: https://www.mindspore.cn/install.\n",
      "[WARNING] ME(1202698:140225058071744,MainProcess):2024-04-26-15:55:16.761.074 [mindspore/run_check/_check_version.py:102] MindSpore version 2.3.0rc1 and cuda version 12.2.140 does not match, CUDA version [['10.1', '11.1', '11.6']] are supported by MindSpore officially. Please refer to the installation guide for version matching information: https://www.mindspore.cn/install.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'\n",
    "os.environ[\"PATH\"] = \"/home/daiyuxin/miniconda3/envs/lyk_ms2.3/bin:/usr/local/cuda-11.6/bin:/home/daiyuxin/.vscode-server/bin/8b3775030ed1a69b13e4f4c628c612102e30a681/bin/remote-cli:/home/daiyuxin/.local/bin:/home/daiyuxin/bin:/usr/local/cuda-11.6/bin:/home/daiyuxin/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin\"\n",
    "os.environ[\"LD_LIBRARY_PATH\"] = \"/usr/local/cuda-11.6/lib64\"\n",
    "os.environ[\"CUDA_HOME\"] = \"/usr/local/cuda-11.6\"\n",
    "import mindspore\n",
    "import datasets\n",
    "from tqdm import tqdm\n",
    "from mindnlp.transformers import BartTokenizer, AutoModelForSeq2SeqLM\n",
    "import numpy as np\n",
    "\n",
    "from mindnlp.peft import AdaLoraConfig, PeftConfig, PeftModel, TaskType, get_peft_model\n",
    "from mindspore.dataset import GeneratorDataset\n",
    "\n",
    "mindspore.set_context(device_target=\"GPU\")\n",
    "\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "model_name_or_path = \"facebook/bart-base\"\n",
    "tokenizer_name_or_path = \"facebook/bart-base\"\n",
    "\n",
    "checkpoint_name = \"financial_sentiment_analysis_lora_v1.pt\"\n",
    "text_column = \"sentence\"\n",
    "label_column = \"text_label\"\n",
    "max_length = 128\n",
    "lr = 1e-3\n",
    "num_epochs = 8\n",
    "batch_size = 8\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating model\n",
    "peft_config = AdaLoraConfig(\n",
    "    init_r=12,\n",
    "    target_r=8,\n",
    "    beta1=0.85,\n",
    "    beta2=0.85,\n",
    "    tinit=200,\n",
    "    tfinal=1000,\n",
    "    deltaT=10,\n",
    "    lora_alpha=32,\n",
    "    lora_dropout=0.1,\n",
    "    task_type=TaskType.SEQ_2_SEQ_LM,\n",
    "    inference_mode=False,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 2,434,176 || all params: 141,854,688 || trainable%: 1.715964438200308\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n",
    "model = get_peft_model(model, peft_config)\n",
    "model.print_trainable_parameters()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-04-26 15:55:22--  https://hf-mirror.com/datasets/financial_phrasebank/resolve/main/data/FinancialPhraseBank-v1.0.zip\n",
      "Resolving hf-mirror.com (hf-mirror.com)... 153.121.57.40, 160.16.199.204, 133.242.169.68\n",
      "Connecting to hf-mirror.com (hf-mirror.com)|153.121.57.40|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://cdn-lfs.hf-mirror.com/datasets/financial_phrasebank/0e1a06c4900fdae46091d031068601e3773ba067c7cecb5b0da1dcba5ce989a6?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27FinancialPhraseBank-v1.0.zip%3B+filename%3D%22FinancialPhraseBank-v1.0.zip%22%3B&response-content-type=application%2Fzip&Expires=1714376978&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNDM3Njk3OH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9kYXRhc2V0cy9maW5hbmNpYWxfcGhyYXNlYmFuay8wZTFhMDZjNDkwMGZkYWU0NjA5MWQwMzEwNjg2MDFlMzc3M2JhMDY3YzdjZWNiNWIwZGExZGNiYTVjZTk4OWE2P3Jlc3BvbnNlLWNvbnRlbnQtZGlzcG9zaXRpb249KiZyZXNwb25zZS1jb250ZW50LXR5cGU9KiJ9XX0_&Signature=NZDzrPidIsMMb9%7EHgKznYgGQOC9XDSrqxBmn9bRGtwdIZVFNrFBNW7gwUqYMp1jAWJNP7kXZHRrzqaIirdoLfDPKifpmUkW%7EOb6Yz3U9VSD5ONAWb2BHuntyGGV5vWPpvxh3-2j1dt3Xo2fkkq749Uk8nbwfd9P1upnNkQaDNBEmebMhGFV8JNJO3-GagbyVn0c2PGNMFv8t1sN2jvbINYkFeoydhnU%7Em1SJTVWyzUN5A2JZJCtPWV8kBv04m%7EsM9bMsFy5M-t7t1ObRlyy8hM1gyDwzVyxUWcUicPAU8GQ4Sunv%7Erhp-6oT7197VMzG-ENw4zEeROc5DlyNdVNEHg__&Key-Pair-Id=KVTP0A1DKRTAX [following]\n",
      "--2024-04-26 15:55:24--  https://cdn-lfs.hf-mirror.com/datasets/financial_phrasebank/0e1a06c4900fdae46091d031068601e3773ba067c7cecb5b0da1dcba5ce989a6?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27FinancialPhraseBank-v1.0.zip%3B+filename%3D%22FinancialPhraseBank-v1.0.zip%22%3B&response-content-type=application%2Fzip&Expires=1714376978&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcxNDM3Njk3OH19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy5odWdnaW5nZmFjZS5jby9kYXRhc2V0cy9maW5hbmNpYWxfcGhyYXNlYmFuay8wZTFhMDZjNDkwMGZkYWU0NjA5MWQwMzEwNjg2MDFlMzc3M2JhMDY3YzdjZWNiNWIwZGExZGNiYTVjZTk4OWE2P3Jlc3BvbnNlLWNvbnRlbnQtZGlzcG9zaXRpb249KiZyZXNwb25zZS1jb250ZW50LXR5cGU9KiJ9XX0_&Signature=NZDzrPidIsMMb9%7EHgKznYgGQOC9XDSrqxBmn9bRGtwdIZVFNrFBNW7gwUqYMp1jAWJNP7kXZHRrzqaIirdoLfDPKifpmUkW%7EOb6Yz3U9VSD5ONAWb2BHuntyGGV5vWPpvxh3-2j1dt3Xo2fkkq749Uk8nbwfd9P1upnNkQaDNBEmebMhGFV8JNJO3-GagbyVn0c2PGNMFv8t1sN2jvbINYkFeoydhnU%7Em1SJTVWyzUN5A2JZJCtPWV8kBv04m%7EsM9bMsFy5M-t7t1ObRlyy8hM1gyDwzVyxUWcUicPAU8GQ4Sunv%7Erhp-6oT7197VMzG-ENw4zEeROc5DlyNdVNEHg__&Key-Pair-Id=KVTP0A1DKRTAX\n",
      "Resolving cdn-lfs.hf-mirror.com (cdn-lfs.hf-mirror.com)... 162.159.33.143, 104.21.63.170, 162.159.142.99, ...\n",
      "Connecting to cdn-lfs.hf-mirror.com (cdn-lfs.hf-mirror.com)|162.159.33.143|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 681890 (666K) [application/zip]\n",
      "Saving to: ‘FinancialPhraseBank-v1.0.zip’\n",
      "\n",
      "FinancialPhraseBank 100%[===================>] 665.91K  1.84MB/s    in 0.4s    \n",
      "\n",
      "2024-04-26 15:55:26 (1.84 MB/s) - ‘FinancialPhraseBank-v1.0.zip’ saved [681890/681890]\n",
      "\n",
      "Archive:  FinancialPhraseBank-v1.0.zip\n",
      "   creating: FinancialPhraseBank-v1.0/\n",
      "  inflating: FinancialPhraseBank-v1.0/License.txt  \n",
      "   creating: __MACOSX/\n",
      "   creating: __MACOSX/FinancialPhraseBank-v1.0/\n",
      "  inflating: __MACOSX/FinancialPhraseBank-v1.0/._License.txt  \n",
      "  inflating: FinancialPhraseBank-v1.0/README.txt  \n",
      "  inflating: __MACOSX/FinancialPhraseBank-v1.0/._README.txt  \n",
      "  inflating: FinancialPhraseBank-v1.0/Sentences_50Agree.txt  \n",
      "  inflating: FinancialPhraseBank-v1.0/Sentences_66Agree.txt  \n",
      "  inflating: FinancialPhraseBank-v1.0/Sentences_75Agree.txt  \n",
      "  inflating: FinancialPhraseBank-v1.0/Sentences_AllAgree.txt  \n"
     ]
    }
   ],
   "source": [
    "# download dataset\n",
    "!wget https://hf-mirror.com/datasets/financial_phrasebank/resolve/main/data/FinancialPhraseBank-v1.0.zip\n",
    "!unzip FinancialPhraseBank-v1.0.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_dataset(source,   batch_size=32, shuffle=False):\n",
    "\n",
    "    column_names = ['input_ids', 'attention_mask','labels','text_labels']\n",
    "    \n",
    "    dataset = GeneratorDataset(source, column_names=column_names, shuffle=shuffle)\n",
    "    # transforms\n",
    "\n",
    "    dataset = dataset.batch(batch_size)\n",
    "\n",
    "    return dataset\n",
    "\n",
    "class MSDataset:\n",
    "    def __init__(self, filepath,tokenizer,max_length):\n",
    "        self.path = filepath\n",
    "        self.sentences = []\n",
    "        self.labels = []\n",
    "        self.text_labels = []\n",
    "        self._load()\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_length = max_length\n",
    "\n",
    "    def _load(self):\n",
    "        label_mapping = {\n",
    "            \"negative\": 0,\n",
    "            \"neutral\": 1,\n",
    "            \"positive\": 2\n",
    "        }\n",
    "        with open(self.path, encoding=\"iso-8859-1\") as f:\n",
    "            for line in f:\n",
    "                sentence, label_text = line.strip().split(\"@\")\n",
    "                self.sentences.append(sentence)\n",
    "                self.labels.append(label_mapping[label_text])\n",
    "                self.text_labels.append(label_text)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        sentence = self.sentences[index]\n",
    "        text_labels = self.text_labels[index]\n",
    "        model_inputs = self.tokenizer(sentence, max_length=self.max_length, padding=\"max_length\", truncation=True, return_tensors=\"np\")\n",
    "        labels = self.tokenizer(text_labels, max_length=3, padding=\"max_length\", truncation=True, return_tensors=\"np\")\n",
    "        labels = labels[\"input_ids\"]\n",
    "        labels[labels == self.tokenizer.pad_token_id] = -100\n",
    "        \n",
    "        return model_inputs['input_ids'], model_inputs['attention_mask'], labels,self.text_labels[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.sentences)\n",
    "    \n",
    "tokenizer = BartTokenizer.from_pretrained(model_name_or_path)\n",
    "dataset = process_dataset(MSDataset(\"./FinancialPhraseBank-v1.0/Sentences_AllAgree.txt\",tokenizer,max_length),batch_size=batch_size)\n",
    "\n",
    "train_dataset, eval_dataset = dataset.split([0.9, 0.1])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = mindspore.nn.AdamWeightDecay(model.trainable_params(), learning_rate=lr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': Tensor(shape=[8, 1, 128], dtype=Int64, value=\n",
      "[[[    0,   597,  5246 ...     1,     1,     1]],\n",
      " [[    0,   597,  5246 ...     1,     1,     1]],\n",
      " [[    0,  1121,   644 ...     1,     1,     1]],\n",
      " ...\n",
      " [[    0, 20420,  1295 ...     1,     1,     1]],\n",
      " [[    0, 20420,  1295 ...     1,     1,     1]],\n",
      " [[    0, 20420,  1295 ...     1,     1,     1]]]), 'attention_mask': Tensor(shape=[8, 1, 128], dtype=Int64, value=\n",
      "[[[1, 1, 1 ... 0, 0, 0]],\n",
      " [[1, 1, 1 ... 0, 0, 0]],\n",
      " [[1, 1, 1 ... 0, 0, 0]],\n",
      " ...\n",
      " [[1, 1, 1 ... 0, 0, 0]],\n",
      " [[1, 1, 1 ... 0, 0, 0]],\n",
      " [[1, 1, 1 ... 0, 0, 0]]]), 'labels': Tensor(shape=[8, 1, 3], dtype=Int64, value=\n",
      "[[[    0, 33407,     2]],\n",
      " [[    0, 33407,     2]],\n",
      " [[    0, 33407,     2]],\n",
      " ...\n",
      " [[    0, 33407,     2]],\n",
      " [[    0, 33407,     2]],\n",
      " [[    0, 33407,     2]]]), 'text_labels': Tensor(shape=[8], dtype=String, value= ['negative', 'negative', 'negative', 'negative', 'negative', 'negative',\n",
      " 'negative', 'negative'])}\n"
     ]
    }
   ],
   "source": [
    "print(next(train_dataset.create_dict_iterator()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:26<00:00,  1.74it/s, train-loss=0.62]\n",
      "100%|██████████| 28/28 [00:05<00:00,  5.38it/s, eval-loss=5.66] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=93.75 % on the evaluation dataset\n",
      "epoch=0: train_ppl=1.8609879621920542 train_epoch_loss=0.6211075091771051 eval_ppl=1.2238705103251357 eval_epoch_loss=0.20201838627571658\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:25<00:00,  1.75it/s, train-loss=0.13]\n",
      "100%|██████████| 28/28 [00:05<00:00,  4.79it/s, eval-loss=1.16]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=97.32142857142857 % on the evaluation dataset\n",
      "epoch=1: train_ppl=1.1363381974778828 train_epoch_loss=0.12781098503984656 eval_ppl=1.042447355381869 eval_epoch_loss=0.041571174981072545\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:24<00:00,  1.76it/s, train-loss=0.09]\n",
      "100%|██████████| 28/28 [00:05<00:00,  5.36it/s, eval-loss=0.86]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=98.21428571428571 % on the evaluation dataset\n",
      "epoch=2: train_ppl=1.0987274775015643 train_epoch_loss=0.09415267151506508 eval_ppl=1.031097363727399 eval_epoch_loss=0.030623636781047955\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:24<00:00,  1.77it/s, train-loss=0.07]\n",
      "100%|██████████| 28/28 [00:05<00:00,  4.83it/s, eval-loss=0.67]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=98.21428571428571 % on the evaluation dataset\n",
      "epoch=3: train_ppl=1.0760787627259034 train_epoch_loss=0.0733236586200256 eval_ppl=1.0241355532391463 eval_epoch_loss=0.023848894066462407\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:35<00:00,  1.64it/s, train-loss=0.07]\n",
      "100%|██████████| 28/28 [00:05<00:00,  5.00it/s, eval-loss=0.60]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=99.10714285714286 % on the evaluation dataset\n",
      "epoch=4: train_ppl=1.06777834246325 train_epoch_loss=0.06558017448759547 eval_ppl=1.0215594030444257 eval_epoch_loss=0.0213302863448174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:32<00:00,  1.67it/s, train-loss=0.08]\n",
      "100%|██████████| 28/28 [00:05<00:00,  4.97it/s, eval-loss=0.74]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=94.19642857142857 % on the evaluation dataset\n",
      "epoch=5: train_ppl=1.0851041400075039 train_epoch_loss=0.08167596396022275 eval_ppl=1.0269215330557264 eval_epoch_loss=0.026565523992164285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:37<00:00,  1.62it/s, train-loss=0.04]\n",
      "100%|██████████| 28/28 [00:05<00:00,  5.27it/s, eval-loss=0.32]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=97.32142857142857 % on the evaluation dataset\n",
      "epoch=6: train_ppl=1.0361444693679083 train_epoch_loss=0.03550658331197851 eval_ppl=1.0116156410396517 eval_epoch_loss=0.011548697378258315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 255/255 [02:34<00:00,  1.65it/s, train-loss=0.06]\n",
      "100%|██████████| 28/28 [00:04<00:00,  5.60it/s, eval-loss=0.55]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy=97.32142857142857 % on the evaluation dataset\n",
      "epoch=7: train_ppl=1.0618449770364131 train_epoch_loss=0.06000793950595692 eval_ppl=1.0197096077598435 eval_epoch_loss=0.019517888487269153\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model.base_model.peft_config[\"default\"].total_step = len(train_dataset) * num_epochs\n",
    "from mindspore import Tensor\n",
    "\n",
    "num_batches = len(train_dataset)\n",
    "num_batches_eval = len(eval_dataset)\n",
    "                       \n",
    "def forward_fn(input_ids,attention_mask,labels ):\n",
    "        output = model(\n",
    "            input_ids=input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            labels=labels,\n",
    "        )\n",
    "        return output.loss, output.logits\n",
    "    \n",
    "grad_fn = mindspore.value_and_grad(\n",
    "        forward_fn, None,model.trainable_params(), has_aux=True,return_ids=True\n",
    "    )\n",
    "global_step = 0\n",
    "for epoch in range(num_epochs):\n",
    "    model.set_train(True)\n",
    "    total_loss, total_step = 0, 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with tqdm(total=num_batches) as t:\n",
    "        for step, (input_ids,attention_mask,labels,_) in enumerate(train_dataset):\n",
    "            input_ids  = input_ids.squeeze(axis=1)\n",
    "            labels  = labels.squeeze(axis=1)\n",
    "            attention_mask = attention_mask.squeeze(axis=1)\n",
    "            (loss, logits), grad = grad_fn(input_ids,attention_mask,labels)\n",
    "            gradient = [g for _, g in grad]\n",
    "            gradient = tuple(gradient)\n",
    "            optimizer(gradient)\n",
    "            model.base_model.update_and_allocate(global_step, grad)\n",
    "            total_loss += loss.asnumpy()\n",
    "            total_step += 1\n",
    "            global_step += 1\n",
    "            curr_loss = total_loss / total_step\n",
    "            t.set_postfix({'train-loss': f'{curr_loss:.2f}'})\n",
    "            t.update(1)\n",
    "    model.set_train(False)\n",
    "    eval_loss = 0\n",
    "    total_step = 0\n",
    "    eval_preds = []\n",
    "    text_labels = []\n",
    "    with tqdm(total=num_batches_eval) as t:\n",
    "        for step, (input_ids,attention_mask,labels,text) in enumerate(eval_dataset):\n",
    "            input_ids = input_ids.squeeze(axis=1)\n",
    "            labels = labels.squeeze(axis=1)\n",
    "            attention_mask = attention_mask.squeeze(axis=1)\n",
    "            outputs = model(input_ids=input_ids,attention_mask=attention_mask,labels=labels)\n",
    "            loss = outputs.loss\n",
    "            eval_loss += loss.asnumpy()\n",
    "            total_step += 1           \n",
    "            eval_loss = total_loss / total_step\n",
    "            eval_preds.extend(\n",
    "                tokenizer.batch_decode(np.argmax(outputs.logits.asnumpy(), -1), skip_special_tokens=True)\n",
    "            )\n",
    "            text_str = str(text.asnumpy())\n",
    "            text_str = text_str.replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\").replace(\"'\", \"\")\n",
    "            labels = text_str.split(\" \")\n",
    "            text_labels.extend(labels)\n",
    "            t.set_postfix({'eval-loss': f'{eval_loss:.2f}'})\n",
    "            t.update(1)\n",
    "    for pred, text_label in zip(eval_preds, text_labels):\n",
    "        if pred.strip() == text_label.strip():\n",
    "            correct += 1\n",
    "        total += 1\n",
    "    accuracy = correct / total * 100\n",
    "    print(f\"{accuracy=} % on the evaluation dataset\")\n",
    "    eval_epoch_loss = eval_loss / eval_dataset.get_dataset_size()\n",
    "    eval_ppl = np.exp(eval_epoch_loss)\n",
    "    train_epoch_loss = total_loss / train_dataset.get_dataset_size()\n",
    "    train_ppl = np.exp(train_epoch_loss)\n",
    "    print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",
    "model.save_pretrained(peft_model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ckpt = f\"{peft_model_id}/adapter_model.ckpt\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING] ME(1202698:140225058071744,MainProcess):2024-04-26-16:16:16.884.580 [mindspore/train/serialization.py:1456] For 'load_param_into_net', 643 parameters in the 'net' are not loaded, because they are not in the 'parameter_dict', please check whether the network structure is consistent when training and loading checkpoint.\n",
      "[WARNING] ME(1202698:140225058071744,MainProcess):2024-04-26-16:16:16.885.453 [mindspore/train/serialization.py:1460] ['base_model.model.shared.weight', 'base_model.model.encoder.embed_positions.weight', 'base_model.model.encoder.layers.0.self_attn.k_proj.weight', 'base_model.model.encoder.layers.0.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.0.self_attn.v_proj.weight', 'base_model.model.encoder.layers.0.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.0.self_attn.q_proj.weight', 'base_model.model.encoder.layers.0.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.0.self_attn.out_proj.weight', 'base_model.model.encoder.layers.0.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.0.self_attn_layer_norm.weight', 'base_model.model.encoder.layers.0.self_attn_layer_norm.bias', 'base_model.model.encoder.layers.0.fc1.weight', 'base_model.model.encoder.layers.0.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.0.fc2.weight', 'base_model.model.encoder.layers.0.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.0.final_layer_norm.weight', 'base_model.model.encoder.layers.0.final_layer_norm.bias', 'base_model.model.encoder.layers.1.self_attn.k_proj.weight', 'base_model.model.encoder.layers.1.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.1.self_attn.v_proj.weight', 'base_model.model.encoder.layers.1.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.1.self_attn.q_proj.weight', 'base_model.model.encoder.layers.1.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.1.self_attn.out_proj.weight', 'base_model.model.encoder.layers.1.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.1.self_attn_layer_norm.weight', 'base_model.model.encoder.layers.1.self_attn_layer_norm.bias', 'base_model.model.encoder.layers.1.fc1.weight', 'base_model.model.encoder.layers.1.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.1.fc2.weight', 'base_model.model.encoder.layers.1.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.1.final_layer_norm.weight', 'base_model.model.encoder.layers.1.final_layer_norm.bias', 'base_model.model.encoder.layers.2.self_attn.k_proj.weight', 'base_model.model.encoder.layers.2.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.2.self_attn.v_proj.weight', 'base_model.model.encoder.layers.2.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.2.self_attn.q_proj.weight', 'base_model.model.encoder.layers.2.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.2.self_attn.out_proj.weight', 'base_model.model.encoder.layers.2.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.2.self_attn_layer_norm.weight', 'base_model.model.encoder.layers.2.self_attn_layer_norm.bias', 'base_model.model.encoder.layers.2.fc1.weight', 'base_model.model.encoder.layers.2.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.2.fc2.weight', 'base_model.model.encoder.layers.2.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.2.final_layer_norm.weight', 'base_model.model.encoder.layers.2.final_layer_norm.bias', 'base_model.model.encoder.layers.3.self_attn.k_proj.weight', 'base_model.model.encoder.layers.3.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.3.self_attn.v_proj.weight', 'base_model.model.encoder.layers.3.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.3.self_attn.q_proj.weight', 'base_model.model.encoder.layers.3.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.3.self_attn.out_proj.weight', 'base_model.model.encoder.layers.3.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.3.self_attn_layer_norm.weight', 'base_model.model.encoder.layers.3.self_attn_layer_norm.bias', 'base_model.model.encoder.layers.3.fc1.weight', 'base_model.model.encoder.layers.3.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.3.fc2.weight', 'base_model.model.encoder.layers.3.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.3.final_layer_norm.weight', 'base_model.model.encoder.layers.3.final_layer_norm.bias', 'base_model.model.encoder.layers.4.self_attn.k_proj.weight', 'base_model.model.encoder.layers.4.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.4.self_attn.v_proj.weight', 'base_model.model.encoder.layers.4.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.4.self_attn.q_proj.weight', 'base_model.model.encoder.layers.4.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.4.self_attn.out_proj.weight', 'base_model.model.encoder.layers.4.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.4.self_attn_layer_norm.weight', 'base_model.model.encoder.layers.4.self_attn_layer_norm.bias', 'base_model.model.encoder.layers.4.fc1.weight', 'base_model.model.encoder.layers.4.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.4.fc2.weight', 'base_model.model.encoder.layers.4.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.4.final_layer_norm.weight', 'base_model.model.encoder.layers.4.final_layer_norm.bias', 'base_model.model.encoder.layers.5.self_attn.k_proj.weight', 'base_model.model.encoder.layers.5.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.5.self_attn.v_proj.weight', 'base_model.model.encoder.layers.5.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.5.self_attn.q_proj.weight', 'base_model.model.encoder.layers.5.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.5.self_attn.out_proj.weight', 'base_model.model.encoder.layers.5.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.5.self_attn_layer_norm.weight', 'base_model.model.encoder.layers.5.self_attn_layer_norm.bias', 'base_model.model.encoder.layers.5.fc1.weight', 'base_model.model.encoder.layers.5.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.5.fc2.weight', 'base_model.model.encoder.layers.5.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.encoder.layers.5.final_layer_norm.weight', 'base_model.model.encoder.layers.5.final_layer_norm.bias', 'base_model.model.encoder.layernorm_embedding.weight', 'base_model.model.encoder.layernorm_embedding.bias', 'base_model.model.decoder.embed_positions.weight', 'base_model.model.decoder.layers.0.self_attn.k_proj.weight', 'base_model.model.decoder.layers.0.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.self_attn.v_proj.weight', 'base_model.model.decoder.layers.0.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.self_attn.q_proj.weight', 'base_model.model.decoder.layers.0.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.self_attn.out_proj.weight', 'base_model.model.decoder.layers.0.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.self_attn_layer_norm.weight', 'base_model.model.decoder.layers.0.self_attn_layer_norm.bias', 'base_model.model.decoder.layers.0.encoder_attn.k_proj.weight', 'base_model.model.decoder.layers.0.encoder_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.encoder_attn.v_proj.weight', 'base_model.model.decoder.layers.0.encoder_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.encoder_attn.q_proj.weight', 'base_model.model.decoder.layers.0.encoder_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.encoder_attn.out_proj.weight', 'base_model.model.decoder.layers.0.encoder_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.encoder_attn_layer_norm.weight', 'base_model.model.decoder.layers.0.encoder_attn_layer_norm.bias', 'base_model.model.decoder.layers.0.fc1.weight', 'base_model.model.decoder.layers.0.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.fc2.weight', 'base_model.model.decoder.layers.0.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.0.final_layer_norm.weight', 'base_model.model.decoder.layers.0.final_layer_norm.bias', 'base_model.model.decoder.layers.1.self_attn.k_proj.weight', 'base_model.model.decoder.layers.1.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.self_attn.v_proj.weight', 'base_model.model.decoder.layers.1.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.self_attn.q_proj.weight', 'base_model.model.decoder.layers.1.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.self_attn.out_proj.weight', 'base_model.model.decoder.layers.1.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.self_attn_layer_norm.weight', 'base_model.model.decoder.layers.1.self_attn_layer_norm.bias', 'base_model.model.decoder.layers.1.encoder_attn.k_proj.weight', 'base_model.model.decoder.layers.1.encoder_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.encoder_attn.v_proj.weight', 'base_model.model.decoder.layers.1.encoder_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.encoder_attn.q_proj.weight', 'base_model.model.decoder.layers.1.encoder_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.encoder_attn.out_proj.weight', 'base_model.model.decoder.layers.1.encoder_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.encoder_attn_layer_norm.weight', 'base_model.model.decoder.layers.1.encoder_attn_layer_norm.bias', 'base_model.model.decoder.layers.1.fc1.weight', 'base_model.model.decoder.layers.1.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.fc2.weight', 'base_model.model.decoder.layers.1.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.1.final_layer_norm.weight', 'base_model.model.decoder.layers.1.final_layer_norm.bias', 'base_model.model.decoder.layers.2.self_attn.k_proj.weight', 'base_model.model.decoder.layers.2.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.self_attn.v_proj.weight', 'base_model.model.decoder.layers.2.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.self_attn.q_proj.weight', 'base_model.model.decoder.layers.2.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.self_attn.out_proj.weight', 'base_model.model.decoder.layers.2.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.self_attn_layer_norm.weight', 'base_model.model.decoder.layers.2.self_attn_layer_norm.bias', 'base_model.model.decoder.layers.2.encoder_attn.k_proj.weight', 'base_model.model.decoder.layers.2.encoder_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.encoder_attn.v_proj.weight', 'base_model.model.decoder.layers.2.encoder_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.encoder_attn.q_proj.weight', 'base_model.model.decoder.layers.2.encoder_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.encoder_attn.out_proj.weight', 'base_model.model.decoder.layers.2.encoder_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.encoder_attn_layer_norm.weight', 'base_model.model.decoder.layers.2.encoder_attn_layer_norm.bias', 'base_model.model.decoder.layers.2.fc1.weight', 'base_model.model.decoder.layers.2.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.fc2.weight', 'base_model.model.decoder.layers.2.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.2.final_layer_norm.weight', 'base_model.model.decoder.layers.2.final_layer_norm.bias', 'base_model.model.decoder.layers.3.self_attn.k_proj.weight', 'base_model.model.decoder.layers.3.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.self_attn.v_proj.weight', 'base_model.model.decoder.layers.3.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.self_attn.q_proj.weight', 'base_model.model.decoder.layers.3.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.self_attn.out_proj.weight', 'base_model.model.decoder.layers.3.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.self_attn_layer_norm.weight', 'base_model.model.decoder.layers.3.self_attn_layer_norm.bias', 'base_model.model.decoder.layers.3.encoder_attn.k_proj.weight', 'base_model.model.decoder.layers.3.encoder_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.encoder_attn.v_proj.weight', 'base_model.model.decoder.layers.3.encoder_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.encoder_attn.q_proj.weight', 'base_model.model.decoder.layers.3.encoder_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.encoder_attn.out_proj.weight', 'base_model.model.decoder.layers.3.encoder_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.encoder_attn_layer_norm.weight', 'base_model.model.decoder.layers.3.encoder_attn_layer_norm.bias', 'base_model.model.decoder.layers.3.fc1.weight', 'base_model.model.decoder.layers.3.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.fc2.weight', 'base_model.model.decoder.layers.3.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.3.final_layer_norm.weight', 'base_model.model.decoder.layers.3.final_layer_norm.bias', 'base_model.model.decoder.layers.4.self_attn.k_proj.weight', 'base_model.model.decoder.layers.4.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.self_attn.v_proj.weight', 'base_model.model.decoder.layers.4.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.self_attn.q_proj.weight', 'base_model.model.decoder.layers.4.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.self_attn.out_proj.weight', 'base_model.model.decoder.layers.4.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.self_attn_layer_norm.weight', 'base_model.model.decoder.layers.4.self_attn_layer_norm.bias', 'base_model.model.decoder.layers.4.encoder_attn.k_proj.weight', 'base_model.model.decoder.layers.4.encoder_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.encoder_attn.v_proj.weight', 'base_model.model.decoder.layers.4.encoder_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.encoder_attn.q_proj.weight', 'base_model.model.decoder.layers.4.encoder_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.encoder_attn.out_proj.weight', 'base_model.model.decoder.layers.4.encoder_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.encoder_attn_layer_norm.weight', 'base_model.model.decoder.layers.4.encoder_attn_layer_norm.bias', 'base_model.model.decoder.layers.4.fc1.weight', 'base_model.model.decoder.layers.4.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.fc2.weight', 'base_model.model.decoder.layers.4.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.4.final_layer_norm.weight', 'base_model.model.decoder.layers.4.final_layer_norm.bias', 'base_model.model.decoder.layers.5.self_attn.k_proj.weight', 'base_model.model.decoder.layers.5.self_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.self_attn.v_proj.weight', 'base_model.model.decoder.layers.5.self_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.self_attn.q_proj.weight', 'base_model.model.decoder.layers.5.self_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.self_attn.out_proj.weight', 'base_model.model.decoder.layers.5.self_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.self_attn_layer_norm.weight', 'base_model.model.decoder.layers.5.self_attn_layer_norm.bias', 'base_model.model.decoder.layers.5.encoder_attn.k_proj.weight', 'base_model.model.decoder.layers.5.encoder_attn.k_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.encoder_attn.v_proj.weight', 'base_model.model.decoder.layers.5.encoder_attn.v_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.encoder_attn.q_proj.weight', 'base_model.model.decoder.layers.5.encoder_attn.q_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.encoder_attn.out_proj.weight', 'base_model.model.decoder.layers.5.encoder_attn.out_proj.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.encoder_attn_layer_norm.weight', 'base_model.model.decoder.layers.5.encoder_attn_layer_norm.bias', 'base_model.model.decoder.layers.5.fc1.weight', 'base_model.model.decoder.layers.5.fc1.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.fc2.weight', 'base_model.model.decoder.layers.5.fc2.bias', 'default', 'default', 'default', 'default', 'base_model.model.decoder.layers.5.final_layer_norm.weight', 'base_model.model.decoder.layers.5.final_layer_norm.bias', 'base_model.model.decoder.layernorm_embedding.weight', 'base_model.model.decoder.layernorm_embedding.bias'] are not loaded.\n"
     ]
    }
   ],
   "source": [
    "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",
    "\n",
    "new_config = PeftConfig.from_pretrained(peft_model_id)\n",
    "new_model = AutoModelForSeq2SeqLM.from_pretrained(new_config.base_model_name_or_path)\n",
    "new_model = PeftModel.from_pretrained(new_model, peft_model_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='negative' text_label='negative'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='positive' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='positive' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='positive' text_label='positive'\n",
      "pred='negative' text_label='positive'\n",
      "pred='negative' text_label='positive'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='negative' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='positive' text_label='positive'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='negative' text_label='negative'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n",
      "pred='neutral' text_label='neutral'\n"
     ]
    }
   ],
   "source": [
    "eval_preds = []\n",
    "text_labels = []\n",
    "for step, (input_ids,attention_mask,labels,text) in enumerate(eval_dataset):\n",
    "            input_ids = input_ids.squeeze(axis=1)\n",
    "            labels = labels.squeeze(axis=1)\n",
    "            attention_mask = attention_mask.squeeze(axis=1)\n",
    "            outputs = model(input_ids=input_ids,attention_mask=attention_mask,labels=labels)\n",
    "            eval_preds.extend(\n",
    "                tokenizer.batch_decode(np.argmax(outputs.logits.asnumpy(), -1), skip_special_tokens=True)\n",
    "            )\n",
    "            text_str = str(text.asnumpy())\n",
    "            text_str = text_str.replace(\"[\", \"\").replace(\"]\", \"\").replace(\",\", \"\").replace(\"'\", \"\").replace(\"\\n\", \"\")\n",
    "            labels = text_str.split(\" \")\n",
    "            text_labels.extend(labels)\n",
    "for pred, text_label in zip(eval_preds, text_labels):\n",
    "    print(f\"{pred=} {text_label=}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lyk_ms2.3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
